import json


def load_jiegou():
    hz_jg = {}
    bert_vocab = set( [line.strip() for line in open('D:/workspace/res_input_output/resource/vocab.txt', 'r', encoding='utf-8')])
    with open('D:/workspace/res_input_output/resource/zh_jiegou.json', 'r', encoding='utf-8') as f:
        js = json.load(f)
        oov = {}
        for k, v in js.items():
            #print(k, v)
            comp = v['components']
            vals = []
            for c in comp:
                if type(c) == dict:
                    for d in c['components']:
                        if type(d) == str:
                            if len(d) > 1:
                                d = d[-1]
                            vals.append(d)
                        else:
                            pass
                elif type(c) == str:
                    if len(c) > 1:
                        c = c[-1]
                    vals.append(c)
            
            has_unk = False
            
            for _, vv in enumerate(vals):
                
                if vv not in bert_vocab:
                    oov[vv] = oov.get(vv, 0) + 1
                    has_unk = True
                    break
                
            if has_unk == True or len(vals) == 0:
                pass
            elif k in bert_vocab:
                hz_jg[k] = vals
                #print(k +'\t'+ ' '.join(vals))
        # kv = sorted(oov.items(), key=lambda x: x[1], reverse=True)
        # for v, k in kv[:80]:
        #     print(v)
        
load_jiegou()